{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2544dd96-a13c-42df-ab63-c427e68fd41e",
   "metadata": {},
   "source": [
    "# Lesson 3: Agentic Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0168aee-bce9-4d60-b827-f86a88187e31",
   "metadata": {
    "height": 182
   },
   "outputs": [],
   "source": [
    "# libraries\n",
    "from dotenv import load_dotenv\n",
    "import os\n",
    "from tavily import TavilyClient\n",
    "\n",
    "# load environment variables from .env file\n",
    "_ = load_dotenv()\n",
    "\n",
    "# connect\n",
    "client = TavilyClient(api_key=os.environ.get(\"TAVILY_API_KEY\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2ba84ec-c172-4de7-ac55-e3158a531b23",
   "metadata": {
    "height": 131
   },
   "outputs": [],
   "source": [
    "# run search\n",
    "result = client.search(\"What is in Nvidia's new Blackwell GPU?\",\n",
    "                       include_answer=True)\n",
    "\n",
    "# print the answer\n",
    "result[\"answer\"]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7f3f33c-c7bd-4b7f-9616-b65eef104514",
   "metadata": {},
   "source": [
    "## Regular search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "876d5092-b8ef-4e38-b4d7-0e80c609bf7a",
   "metadata": {
    "height": 165
   },
   "outputs": [],
   "source": [
    "# choose location (try to change to your own city!)\n",
    "\n",
    "city = \"San Francisco\"\n",
    "\n",
    "query = f\"\"\"\n",
    "    what is the current weather in {city}?\n",
    "    Should I travel there today?\n",
    "    \"weather.com\"\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2128e40f-11ef-41ed-ac1f-7feeb5546224",
   "metadata": {},
   "source": [
    "> Note: search was modified to return expected results in the event of an exception. High volumes of student traffic sometimes cause rate limit exceptions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10084a02-2928-4945-9f7c-ad3f5b33caf7",
   "metadata": {
    "height": 403
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from duckduckgo_search import DDGS\n",
    "import re\n",
    "\n",
    "ddg = DDGS()\n",
    "\n",
    "def search(query, max_results=6):\n",
    "    try:\n",
    "        results = ddg.text(query, max_results=max_results)\n",
    "        return [i[\"href\"] for i in results]\n",
    "    except Exception as e:\n",
    "        print(f\"returning previous results due to exception reaching ddg.\")\n",
    "        results = [ # cover case where DDG rate limits due to high deeplearning.ai volume\n",
    "            \"https://weather.com/weather/today/l/USCA0987:1:US\",\n",
    "            \"https://weather.com/weather/hourbyhour/l/54f9d8baac32496f6b5497b4bf7a277c3e2e6cc5625de69680e6169e7e38e9a8\",\n",
    "        ]\n",
    "        return results  \n",
    "\n",
    "\n",
    "for i in search(query):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d31c432d-8e22-412f-b302-961ace0b00bd",
   "metadata": {
    "height": 267
   },
   "outputs": [],
   "source": [
    "def scrape_weather_info(url):\n",
    "    \"\"\"Scrape content from the given URL\"\"\"\n",
    "    if not url:\n",
    "        return \"Weather information could not be found.\"\n",
    "    \n",
    "    # fetch data\n",
    "    headers = {'User-Agent': 'Mozilla/5.0'}\n",
    "    response = requests.get(url, headers=headers)\n",
    "    if response.status_code != 200:\n",
    "        return \"Failed to retrieve the webpage.\"\n",
    "\n",
    "    # parse result\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    return soup\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b67d8ad-a439-4c91-9dfe-7c84998ef644",
   "metadata": {},
   "source": [
    "> Note: This produces a long output, you may want to right click and clear the cell output after you look at it briefly to avoid scrolling past it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "714d1205-f8fc-4912-b148-2a45da99219c",
   "metadata": {
    "height": 148
   },
   "outputs": [],
   "source": [
    "# use DuckDuckGo to find websites and take the first result\n",
    "url = search(query)[0]\n",
    "\n",
    "# scrape first wesbsite\n",
    "soup = scrape_weather_info(url)\n",
    "\n",
    "print(f\"Website: {url}\\n\\n\")\n",
    "print(str(soup.body)[:50000]) # limit long outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cb3ef4c-58b3-401b-b104-0d51e553d982",
   "metadata": {
    "height": 250
   },
   "outputs": [],
   "source": [
    "# extract text\n",
    "weather_data = []\n",
    "for tag in soup.find_all(['h1', 'h2', 'h3', 'p']):\n",
    "    text = tag.get_text(\" \", strip=True)\n",
    "    weather_data.append(text)\n",
    "\n",
    "# combine all elements into a single string\n",
    "weather_data = \"\\n\".join(weather_data)\n",
    "\n",
    "# remove all spaces from the combined text\n",
    "weather_data = re.sub(r'\\s+', ' ', weather_data)\n",
    "    \n",
    "print(f\"Website: {url}\\n\\n\")\n",
    "print(weather_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92db676d-d8d9-4558-8dcf-1b20fcb48e45",
   "metadata": {},
   "source": [
    "## Agentic Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc3293b7-a50c-43c8-a022-8975e1e444b8",
   "metadata": {
    "height": 131
   },
   "outputs": [],
   "source": [
    "# run search\n",
    "result = client.search(query, max_results=1)\n",
    "\n",
    "# print first result\n",
    "data = result[\"results\"][0][\"content\"]\n",
    "\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0722c3d4-4cbf-43bf-81b0-50f634c4ce61",
   "metadata": {
    "height": 250
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from pygments import highlight, lexers, formatters\n",
    "\n",
    "# parse JSON\n",
    "parsed_json = json.loads(data.replace(\"'\", '\"'))\n",
    "\n",
    "# pretty print JSON with syntax highlighting\n",
    "formatted_json = json.dumps(parsed_json, indent=4)\n",
    "colorful_json = highlight(formatted_json,\n",
    "                          lexers.JsonLexer(),\n",
    "                          formatters.TerminalFormatter())\n",
    "\n",
    "print(colorful_json)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "837f3ecd-df6b-4f2f-9325-a253678897b0",
   "metadata": {},
   "source": [
    "<img src=\"./google_sample.png\" width=\"800\" height=\"600\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61f79eb9-d1c9-44b0-9efd-a8f9b380332a",
   "metadata": {
    "height": 29
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
